{ "description": "Scrape web pages to be used with Retrieval Augmented Generation (RAG) for question answering", "usecases": ["Documents QnA"], "framework": ["Langchain"], "badge": "POPULAR", "nodes": [ { "width": 300, "height": 424, "id": "openAIEmbeddings_0", "position": { "x": 805.4033852865127, "y": 289.17383087232275 }, "type": "customNode", "data": { "id": "openAIEmbeddings_0", "label": "OpenAI Embeddings", "version": 4, "name": "openAIEmbeddings", "type": "OpenAIEmbeddings", "baseClasses": ["OpenAIEmbeddings", "Embeddings"], "category": "Embeddings", "description": "OpenAI API to generate embeddings for a given text", "inputParams": [ { "label": "Connect Credential", "name": "credential", "type": "credential", "credentialNames": ["openAIApi"], "id": "openAIEmbeddings_0-input-credential-credential" }, { "label": "Model Name", "name": "modelName", "type": "asyncOptions", "loadMethod": "listModels", "default": "text-embedding-ada-002", "id": "openAIEmbeddings_0-input-modelName-asyncOptions" }, { "label": "Strip New Lines", "name": "stripNewLines", "type": "boolean", "optional": true, "additionalParams": true, "id": "openAIEmbeddings_0-input-stripNewLines-boolean" }, { "label": "Batch Size", "name": "batchSize", "type": "number", "optional": true, "additionalParams": true, "id": "openAIEmbeddings_0-input-batchSize-number" }, { "label": "Timeout", "name": "timeout", "type": "number", "optional": true, "additionalParams": true, "id": "openAIEmbeddings_0-input-timeout-number" }, { "label": "BasePath", "name": "basepath", "type": "string", "optional": true, "additionalParams": true, "id": "openAIEmbeddings_0-input-basepath-string" }, { "label": "Dimensions", "name": "dimensions", "type": "number", "optional": true, "additionalParams": true, "id": "openAIEmbeddings_0-input-dimensions-number" } ], "inputAnchors": [], "inputs": { "modelName": "text-embedding-ada-002", "stripNewLines": "", "batchSize": "", "timeout": "", "basepath": "", "dimensions": "" }, "outputAnchors": [ { "id": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings", "name": "openAIEmbeddings", "label": "OpenAIEmbeddings", "description": "OpenAI API to generate embeddings for a given text", "type": "OpenAIEmbeddings | Embeddings" } ], "outputs": {}, "selected": false }, "selected": false, "positionAbsolute": { "x": 805.4033852865127, "y": 289.17383087232275 }, "dragging": false }, { "width": 300, "height": 378, "id": "htmlToMarkdownTextSplitter_0", "position": { "x": 459.0189921792261, "y": -21.97787557438943 }, "type": "customNode", "data": { "id": "htmlToMarkdownTextSplitter_0", "label": "HtmlToMarkdown Text Splitter", "version": 1, "name": "htmlToMarkdownTextSplitter", "type": "HtmlToMarkdownTextSplitter", "baseClasses": [ "HtmlToMarkdownTextSplitter", "MarkdownTextSplitter", "RecursiveCharacterTextSplitter", "TextSplitter", "BaseDocumentTransformer" ], "category": "Text Splitters", "description": "Converts Html to Markdown and then split your content into documents based on the Markdown headers", "inputParams": [ { "label": "Chunk Size", "name": "chunkSize", "type": "number", "default": 1000, "optional": true, "id": "htmlToMarkdownTextSplitter_0-input-chunkSize-number" }, { "label": "Chunk Overlap", "name": "chunkOverlap", "type": "number", "optional": true, "id": "htmlToMarkdownTextSplitter_0-input-chunkOverlap-number" } ], "inputAnchors": [], "inputs": { "chunkSize": "4000", "chunkOverlap": "" }, "outputAnchors": [ { "id": "htmlToMarkdownTextSplitter_0-output-htmlToMarkdownTextSplitter-HtmlToMarkdownTextSplitter|MarkdownTextSplitter|RecursiveCharacterTextSplitter|TextSplitter|BaseDocumentTransformer", "name": "htmlToMarkdownTextSplitter", "label": "HtmlToMarkdownTextSplitter", "type": "HtmlToMarkdownTextSplitter | MarkdownTextSplitter | RecursiveCharacterTextSplitter | TextSplitter | BaseDocumentTransformer" } ], "outputs": {}, "selected": false }, "selected": false, "positionAbsolute": { "x": 459.0189921792261, "y": -21.97787557438943 }, "dragging": false }, { "width": 300, "height": 532, "id": "conversationalRetrievalQAChain_0", "position": { "x": 1892.82894546983, "y": 282.2572649522094 }, "type": "customNode", "data": { "id": "conversationalRetrievalQAChain_0", "label": "Conversational Retrieval QA Chain", "version": 3, "name": "conversationalRetrievalQAChain", "type": "ConversationalRetrievalQAChain", "baseClasses": ["ConversationalRetrievalQAChain", "BaseChain", "Runnable"], "category": "Chains", "description": "Document QA - built on RetrievalQAChain to provide a chat history component", "inputParams": [ { "label": "Return Source Documents", "name": "returnSourceDocuments", "type": "boolean", "optional": true, "id": "conversationalRetrievalQAChain_0-input-returnSourceDocuments-boolean" }, { "label": "Rephrase Prompt", "name": "rephrasePrompt", "type": "string", "description": "Using previous chat history, rephrase question into a standalone question", "warning": "Prompt must include input variables: {chat_history} and {question}", "rows": 4, "additionalParams": true, "optional": true, "default": "Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question.\n\nChat History:\n{chat_history}\nFollow Up Input: {question}\nStandalone Question:", "id": "conversationalRetrievalQAChain_0-input-rephrasePrompt-string" }, { "label": "Response Prompt", "name": "responsePrompt", "type": "string", "description": "Taking the rephrased question, search for answer from the provided context", "warning": "Prompt must include input variable: {context}", "rows": 4, "additionalParams": true, "optional": true, "default": "You are a helpful assistant. Using the provided context, answer the user's question to the best of your ability using the resources provided.\nIf there is nothing in the context relevant to the question at hand, just say \"Hmm, I'm not sure.\" Don't try to make up an answer.\n------------\n{context}\n------------\nREMEMBER: If there is no relevant information within the context, just say \"Hmm, I'm not sure.\" Don't try to make up an answer.", "id": "conversationalRetrievalQAChain_0-input-responsePrompt-string" } ], "inputAnchors": [ { "label": "Chat Model", "name": "model", "type": "BaseChatModel", "id": "conversationalRetrievalQAChain_0-input-model-BaseChatModel" }, { "label": "Vector Store Retriever", "name": "vectorStoreRetriever", "type": "BaseRetriever", "id": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever" }, { "label": "Memory", "name": "memory", "type": "BaseMemory", "optional": true, "description": "If left empty, a default BufferMemory will be used", "id": "conversationalRetrievalQAChain_0-input-memory-BaseMemory" }, { "label": "Input Moderation", "description": "Detect text that could generate harmful output and prevent it from being sent to the language model", "name": "inputModeration", "type": "Moderation", "optional": true, "list": true, "id": "conversationalRetrievalQAChain_0-input-inputModeration-Moderation" } ], "inputs": { "inputModeration": "", "model": "{{chatOpenAI_0.data.instance}}", "vectorStoreRetriever": "{{pinecone_0.data.instance}}", "memory": "", "returnSourceDocuments": true, "rephrasePrompt": "Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question.\n\nChat History:\n{chat_history}\nFollow Up Input: {question}\nStandalone Question:", "responsePrompt": "You are a helpful assistant. Using the provided context, answer the user's question to the best of your ability using the resources provided.\nIf there is nothing in the context relevant to the question at hand, just say \"Hmm, I'm not sure.\" Don't try to make up an answer.\n------------\n{context}\n------------\nREMEMBER: If there is no relevant information within the context, just say \"Hmm, I'm not sure.\" Don't try to make up an answer." }, "outputAnchors": [ { "id": "conversationalRetrievalQAChain_0-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|Runnable", "name": "conversationalRetrievalQAChain", "label": "ConversationalRetrievalQAChain", "type": "ConversationalRetrievalQAChain | BaseChain | Runnable" } ], "outputs": {}, "selected": false }, "selected": false, "positionAbsolute": { "x": 1892.82894546983, "y": 282.2572649522094 }, "dragging": false }, { "width": 300, "height": 426, "id": "cheerioWebScraper_0", "position": { "x": 815.9295655148293, "y": -190.50425962124604 }, "type": "customNode", "data": { "id": "cheerioWebScraper_0", "label": "Cheerio Web Scraper", "version": 1.1, "name": "cheerioWebScraper", "type": "Document", "baseClasses": ["Document"], "category": "Document Loaders", "description": "Load data from webpages", "inputParams": [ { "label": "URL", "name": "url", "type": "string", "id": "cheerioWebScraper_0-input-url-string" }, { "label": "Get Relative Links Method", "name": "relativeLinksMethod", "type": "options", "description": "Select a method to retrieve relative links", "options": [ { "label": "Web Crawl", "name": "webCrawl", "description": "Crawl relative links from HTML URL" }, { "label": "Scrape XML Sitemap", "name": "scrapeXMLSitemap", "description": "Scrape relative links from XML sitemap URL" } ], "optional": true, "additionalParams": true, "id": "cheerioWebScraper_0-input-relativeLinksMethod-options" }, { "label": "Get Relative Links Limit", "name": "limit", "type": "number", "optional": true, "additionalParams": true, "description": "Only used when \"Get Relative Links Method\" is selected. Set 0 to retrieve all relative links, default limit is 10.", "warning": "Retrieving all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc)", "id": "cheerioWebScraper_0-input-limit-number" }, { "label": "Selector (CSS)", "name": "selector", "type": "string", "description": "Specify a CSS selector to select the content to be extracted", "optional": true, "additionalParams": true, "id": "cheerioWebScraper_0-input-selector-string" }, { "label": "Metadata", "name": "metadata", "type": "json", "optional": true, "additionalParams": true, "id": "cheerioWebScraper_0-input-metadata-json" } ], "inputAnchors": [ { "label": "Text Splitter", "name": "textSplitter", "type": "TextSplitter", "optional": true, "id": "cheerioWebScraper_0-input-textSplitter-TextSplitter" } ], "inputs": { "url": "https://flowiseai.com/", "textSplitter": "{{htmlToMarkdownTextSplitter_0.data.instance}}", "relativeLinksMethod": "", "limit": "", "metadata": "" }, "outputAnchors": [ { "id": "cheerioWebScraper_0-output-cheerioWebScraper-Document", "name": "cheerioWebScraper", "label": "Document", "type": "Document" } ], "outputs": {}, "selected": false }, "selected": false, "positionAbsolute": { "x": 815.9295655148293, "y": -190.50425962124604 }, "dragging": false }, { "width": 300, "height": 670, "id": "chatOpenAI_0", "position": { "x": 1532.4907022314349, "y": -270.38662863532466 }, "type": "customNode", "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", "version": 6, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], "category": "Chat Models", "description": "Wrapper around OpenAI large language models that use the Chat endpoint", "inputParams": [ { "label": "Connect Credential", "name": "credential", "type": "credential", "credentialNames": ["openAIApi"], "id": "chatOpenAI_0-input-credential-credential" }, { "label": "Model Name", "name": "modelName", "type": "asyncOptions", "loadMethod": "listModels", "default": "gpt-3.5-turbo", "id": "chatOpenAI_0-input-modelName-options" }, { "label": "Temperature", "name": "temperature", "type": "number", "step": 0.1, "default": 0.9, "optional": true, "id": "chatOpenAI_0-input-temperature-number" }, { "label": "Max Tokens", "name": "maxTokens", "type": "number", "step": 1, "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-maxTokens-number" }, { "label": "Top Probability", "name": "topP", "type": "number", "step": 0.1, "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-topP-number" }, { "label": "Frequency Penalty", "name": "frequencyPenalty", "type": "number", "step": 0.1, "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-frequencyPenalty-number" }, { "label": "Presence Penalty", "name": "presencePenalty", "type": "number", "step": 0.1, "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-presencePenalty-number" }, { "label": "Timeout", "name": "timeout", "type": "number", "step": 1, "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-timeout-number" }, { "label": "BasePath", "name": "basepath", "type": "string", "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-basepath-string" }, { "label": "BaseOptions", "name": "baseOptions", "type": "json", "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" }, { "label": "Allow Image Uploads", "name": "allowImageUploads", "type": "boolean", "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", "default": false, "optional": true, "id": "chatOpenAI_0-input-allowImageUploads-boolean" }, { "label": "Image Resolution", "description": "This parameter controls the resolution in which the model views the image.", "name": "imageResolution", "type": "options", "options": [ { "label": "Low", "name": "low" }, { "label": "High", "name": "high" }, { "label": "Auto", "name": "auto" } ], "default": "low", "optional": false, "additionalParams": true, "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ { "label": "Cache", "name": "cache", "type": "BaseCache", "optional": true, "id": "chatOpenAI_0-input-cache-BaseCache" } ], "inputs": { "cache": "", "modelName": "gpt-3.5-turbo", "temperature": 0.9, "maxTokens": "", "topP": "", "frequencyPenalty": "", "presencePenalty": "", "timeout": "", "basepath": "", "baseOptions": "", "allowImageUploads": true, "imageResolution": "low" }, "outputAnchors": [ { "id": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable", "name": "chatOpenAI", "label": "ChatOpenAI", "type": "ChatOpenAI | BaseChatModel | BaseLanguageModel | Runnable" } ], "outputs": {}, "selected": false }, "selected": false, "positionAbsolute": { "x": 1532.4907022314349, "y": -270.38662863532466 }, "dragging": false }, { "width": 300, "height": 555, "id": "pinecone_0", "position": { "x": 1182.9660159923678, "y": 56.45789225898284 }, "type": "customNode", "data": { "id": "pinecone_0", "label": "Pinecone", "version": 3, "name": "pinecone", "type": "Pinecone", "baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"], "category": "Vector Stores", "description": "Upsert embedded data and perform similarity or mmr search using Pinecone, a leading fully managed hosted vector database", "inputParams": [ { "label": "Connect Credential", "name": "credential", "type": "credential", "credentialNames": ["pineconeApi"], "id": "pinecone_0-input-credential-credential" }, { "label": "Pinecone Index", "name": "pineconeIndex", "type": "string", "id": "pinecone_0-input-pineconeIndex-string" }, { "label": "Pinecone Namespace", "name": "pineconeNamespace", "type": "string", "placeholder": "my-first-namespace", "additionalParams": true, "optional": true, "id": "pinecone_0-input-pineconeNamespace-string" }, { "label": "Pinecone Metadata Filter", "name": "pineconeMetadataFilter", "type": "json", "optional": true, "additionalParams": true, "id": "pinecone_0-input-pineconeMetadataFilter-json" }, { "label": "Top K", "name": "topK", "description": "Number of top results to fetch. Default to 4", "placeholder": "4", "type": "number", "additionalParams": true, "optional": true, "id": "pinecone_0-input-topK-number" }, { "label": "Search Type", "name": "searchType", "type": "options", "default": "similarity", "options": [ { "label": "Similarity", "name": "similarity" }, { "label": "Max Marginal Relevance", "name": "mmr" } ], "additionalParams": true, "optional": true, "id": "pinecone_0-input-searchType-options" }, { "label": "Fetch K (for MMR Search)", "name": "fetchK", "description": "Number of initial documents to fetch for MMR reranking. Default to 20. Used only when the search type is MMR", "placeholder": "20", "type": "number", "additionalParams": true, "optional": true, "id": "pinecone_0-input-fetchK-number" }, { "label": "Lambda (for MMR Search)", "name": "lambda", "description": "Number between 0 and 1 that determines the degree of diversity among the results, where 0 corresponds to maximum diversity and 1 to minimum diversity. Used only when the search type is MMR", "placeholder": "0.5", "type": "number", "additionalParams": true, "optional": true, "id": "pinecone_0-input-lambda-number" } ], "inputAnchors": [ { "label": "Document", "name": "document", "type": "Document", "list": true, "optional": true, "id": "pinecone_0-input-document-Document" }, { "label": "Embeddings", "name": "embeddings", "type": "Embeddings", "id": "pinecone_0-input-embeddings-Embeddings" } ], "inputs": { "document": ["{{cheerioWebScraper_0.data.instance}}"], "embeddings": "{{openAIEmbeddings_0.data.instance}}", "pineconeIndex": "", "pineconeNamespace": "", "pineconeMetadataFilter": "", "topK": "", "searchType": "similarity", "fetchK": "", "lambda": "" }, "outputAnchors": [ { "name": "output", "label": "Output", "type": "options", "options": [ { "id": "pinecone_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever", "name": "retriever", "label": "Pinecone Retriever", "type": "Pinecone | VectorStoreRetriever | BaseRetriever" }, { "id": "pinecone_0-output-vectorStore-Pinecone|VectorStore", "name": "vectorStore", "label": "Pinecone Vector Store", "type": "Pinecone | VectorStore" } ], "default": "retriever" } ], "outputs": { "output": "retriever" }, "selected": false }, "selected": false, "positionAbsolute": { "x": 1182.9660159923678, "y": 56.45789225898284 }, "dragging": false } ], "edges": [ { "source": "htmlToMarkdownTextSplitter_0", "sourceHandle": "htmlToMarkdownTextSplitter_0-output-htmlToMarkdownTextSplitter-HtmlToMarkdownTextSplitter|MarkdownTextSplitter|RecursiveCharacterTextSplitter|TextSplitter|BaseDocumentTransformer", "target": "cheerioWebScraper_0", "targetHandle": "cheerioWebScraper_0-input-textSplitter-TextSplitter", "type": "buttonedge", "id": "htmlToMarkdownTextSplitter_0-htmlToMarkdownTextSplitter_0-output-htmlToMarkdownTextSplitter-HtmlToMarkdownTextSplitter|MarkdownTextSplitter|RecursiveCharacterTextSplitter|TextSplitter|BaseDocumentTransformer-cheerioWebScraper_0-cheerioWebScraper_0-input-textSplitter-TextSplitter", "data": { "label": "" } }, { "source": "chatOpenAI_0", "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable", "target": "conversationalRetrievalQAChain_0", "targetHandle": "conversationalRetrievalQAChain_0-input-model-BaseChatModel", "type": "buttonedge", "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-model-BaseChatModel", "data": { "label": "" } }, { "source": "cheerioWebScraper_0", "sourceHandle": "cheerioWebScraper_0-output-cheerioWebScraper-Document", "target": "pinecone_0", "targetHandle": "pinecone_0-input-document-Document", "type": "buttonedge", "id": "cheerioWebScraper_0-cheerioWebScraper_0-output-cheerioWebScraper-Document-pinecone_0-pinecone_0-input-document-Document", "data": { "label": "" } }, { "source": "openAIEmbeddings_0", "sourceHandle": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings", "target": "pinecone_0", "targetHandle": "pinecone_0-input-embeddings-Embeddings", "type": "buttonedge", "id": "openAIEmbeddings_0-openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings-pinecone_0-pinecone_0-input-embeddings-Embeddings", "data": { "label": "" } }, { "source": "pinecone_0", "sourceHandle": "pinecone_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever", "target": "conversationalRetrievalQAChain_0", "targetHandle": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", "type": "buttonedge", "id": "pinecone_0-pinecone_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", "data": { "label": "" } } ] }